{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "pycharm": {
      "stem_cell": {
        "cell_type": "raw",
        "source": [],
        "metadata": {
          "collapsed": false
        }
      }
    },
    "colab": {
      "name": "QA.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ec5480ed053b46cdb517d77899900a2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_85a1571687b84d19ae442c5f81f26f7a",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e2eb771a977f44c180f99f87ca99fd77",
              "IPY_MODEL_cb3ee57a490d4a9592e4b122d0d81948"
            ]
          }
        },
        "85a1571687b84d19ae442c5f81f26f7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e2eb771a977f44c180f99f87ca99fd77": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3da87e6c040b440988d93c43ac3a2c09",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 463,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 463,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dceac99c8e52482e9f62a5a55898641e"
          }
        },
        "cb3ee57a490d4a9592e4b122d0d81948": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a07d84c9acd24a909d65f8b16f85fbe9",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 463/463 [00:00&lt;00:00, 982B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f9e54937bcc49858ae9a938d899379b"
          }
        },
        "3da87e6c040b440988d93c43ac3a2c09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dceac99c8e52482e9f62a5a55898641e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a07d84c9acd24a909d65f8b16f85fbe9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f9e54937bcc49858ae9a938d899379b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "27a07215928f497db5e317b82e9e5922": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7c2581de98954b79b19ee3c6a2259ba7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_7b616c40df534e3fa921221fe620a3d9",
              "IPY_MODEL_23afbc66e2dd43eb84d3d731c46263f1"
            ]
          }
        },
        "7c2581de98954b79b19ee3c6a2259ba7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "7b616c40df534e3fa921221fe620a3d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e084d2d75eb14fe3aacedbd5ecf711bc",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 54236116,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 54236116,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a4b121058c934cf081e8af64e893b913"
          }
        },
        "23afbc66e2dd43eb84d3d731c46263f1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_b7c0ba54037049e7a9ce91a85c41c580",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 54.2M/54.2M [00:02&lt;00:00, 21.2MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d7d6eb6e2945450fb1c0590a506222e5"
          }
        },
        "e084d2d75eb14fe3aacedbd5ecf711bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a4b121058c934cf081e8af64e893b913": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b7c0ba54037049e7a9ce91a85c41c580": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d7d6eb6e2945450fb1c0590a506222e5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a7f35783ec6249be8ccfba1c83ed0e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c9bd7a41ada546e88509a60576dcbd81",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_dad65f326c614a58a0c13c94accab562",
              "IPY_MODEL_f853299184d54861874b30a6087c6e3b"
            ]
          }
        },
        "c9bd7a41ada546e88509a60576dcbd81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dad65f326c614a58a0c13c94accab562": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_863fcf057e7d4fe984c531d6b1291814",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_058caba18b6d4a1e8fb52d573465255e"
          }
        },
        "f853299184d54861874b30a6087c6e3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_6234b7e5ca9c4067ae359a31f9b38e27",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 436kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a475b921ef3d4315990e07512ff759a3"
          }
        },
        "863fcf057e7d4fe984c531d6b1291814": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "058caba18b6d4a1e8fb52d573465255e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6234b7e5ca9c4067ae359a31f9b38e27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a475b921ef3d4315990e07512ff759a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "469aaef964d644198b9cf9b878c56178": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_045cce4661714b078350aa8c12f86680",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4f43fcc08b664d0d9a9c5edbec57dfe2",
              "IPY_MODEL_a61f3ca504574a4db912806d920daad9"
            ]
          }
        },
        "045cce4661714b078350aa8c12f86680": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f43fcc08b664d0d9a9c5edbec57dfe2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_da55a1ee4c1547c180fc2fa62e8908d2",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_9d754136f43641b4996383d99a3163c3"
          }
        },
        "a61f3ca504574a4db912806d920daad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_fd4c6744a60340b4b004710cf2e9c96c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 1.40MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a196782e8624fca9baf0561b48cf0b8"
          }
        },
        "da55a1ee4c1547c180fc2fa62e8908d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "9d754136f43641b4996383d99a3163c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "fd4c6744a60340b4b004710cf2e9c96c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a196782e8624fca9baf0561b48cf0b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eSCTm5d8MhJA"
      },
      "source": [
        "Question Answering Transformers with Hugging Face\n",
        "\n",
        "Copyright 2020 Denis Rothman\n",
        "\n",
        "[Hugging Face notebook Resources and Documentation](https://huggingface.co/)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "name": "#%% code\n"
        },
        "id": "4maAknWNrl_N"
      },
      "source": [
        "!pip install -q transformers==4.0.0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "pycharm": {
          "is_executing": false,
          "name": "#%% code \n"
        },
        "id": "uKaqzCh6rl_V"
      },
      "source": [
        "from transformers import pipeline"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4VYUquAoa2eT"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZxKBah-9iYF7"
      },
      "source": [
        "Sample 1:The traffic began to slow down on Pioneer Boulevard in Los Angeles, making it difficult to get out of the city. However, WBGO was playing some cool jazz, and the weather was cool, making it rather pleasant to be making it out of the city on this Friday afternoon. Nat King Cole was singing as Jo and Maria slowly made their way out of LA and drove toward Barstow. They planned to get to Las Vegas early enough in the evening to have a nice dinner and go see a show."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MqvL7FP6bhzv"
      },
      "source": [
        "sequence = \"The traffic began to slow down on Pioneer Boulevard in Los Angeles, making it difficult to get out of the city. However, WBGO was playing some cool jazz, and the weather was cool, making it rather pleasant to be making it out of the city on this Friday afternoon. Nat King Cole was singing as Jo and Maria slowly made their way out of LA and drove toward Barstow. They planned to get to Las Vegas early enough in the evening to have a nice dinner and go see a show.\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "syYGx5ZF6rkL"
      },
      "source": [
        "Question-Answering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04tFdSHTbsFQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a720d1b-2764-4868-dbb8-e4ed662915ac"
      },
      "source": [
        "nlp_qa(context=sequence, question='Where is Pioneer Boulevard ?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Los Angeles', 'end': 66, 'score': 0.9879737496376038, 'start': 55}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mqt2Z8qN6vNz"
      },
      "source": [
        "Named Entity Recognition(NER)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V5GJSN_ui3J6",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f21cb0f-4d86-4851-bef4-003a6f67ecab"
      },
      "source": [
        "nlp_ner = pipeline(\"ner\")\n",
        "print(nlp_ner(sequence))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[{'word': 'Pioneer', 'score': 0.9735257029533386, 'entity': 'I-LOC', 'index': 8}, {'word': 'Boulevard', 'score': 0.9944824576377869, 'entity': 'I-LOC', 'index': 9}, {'word': 'Los', 'score': 0.9995775818824768, 'entity': 'I-LOC', 'index': 11}, {'word': 'Angeles', 'score': 0.9995693564414978, 'entity': 'I-LOC', 'index': 12}, {'word': 'W', 'score': 0.991984486579895, 'entity': 'I-ORG', 'index': 26}, {'word': '##B', 'score': 0.990750253200531, 'entity': 'I-ORG', 'index': 27}, {'word': '##G', 'score': 0.9884582161903381, 'entity': 'I-ORG', 'index': 28}, {'word': '##O', 'score': 0.9722681641578674, 'entity': 'I-ORG', 'index': 29}, {'word': 'Nat', 'score': 0.9966881275177002, 'entity': 'I-PER', 'index': 59}, {'word': 'King', 'score': 0.997648298740387, 'entity': 'I-PER', 'index': 60}, {'word': 'Cole', 'score': 0.9986170530319214, 'entity': 'I-PER', 'index': 61}, {'word': 'Jo', 'score': 0.9978788495063782, 'entity': 'I-PER', 'index': 65}, {'word': 'Maria', 'score': 0.9988164901733398, 'entity': 'I-PER', 'index': 67}, {'word': 'LA', 'score': 0.998134434223175, 'entity': 'I-LOC', 'index': 74}, {'word': 'Bar', 'score': 0.9970266819000244, 'entity': 'I-LOC', 'index': 78}, {'word': '##sto', 'score': 0.8573915958404541, 'entity': 'I-LOC', 'index': 79}, {'word': '##w', 'score': 0.9920249581336975, 'entity': 'I-LOC', 'index': 80}, {'word': 'Las', 'score': 0.9993551969528198, 'entity': 'I-LOC', 'index': 87}, {'word': 'Vegas', 'score': 0.9989539384841919, 'entity': 'I-LOC', 'index': 88}]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ye1D9aYaun7y",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e49f9baa-b5e4-401c-d051-951ff090a209"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')\n",
        "print(\"Question 1.\",nlp_qa(context=sequence, question='Where is Pioneer Boulevard ?'))\n",
        "print(\"Question 2.\",nlp_qa(context=sequence, question='Where is Los Angeles located?'))\n",
        "print(\"Question 3.\",nlp_qa(context=sequence, question='Where is LA ?'))\n",
        "print(\"Question 4.\",nlp_qa(context=sequence, question='Where is Barstow ?'))\n",
        "print(\"Question 5.\",nlp_qa(context=sequence, question='Where is Las Vegas located ?'))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Question 1. {'score': 0.9879737496376038, 'start': 55, 'end': 66, 'answer': 'Los Angeles'}\n",
            "Question 2. {'score': 0.9875388741493225, 'start': 34, 'end': 51, 'answer': 'Pioneer Boulevard'}\n",
            "Question 3. {'score': 0.5090540647506714, 'start': 55, 'end': 66, 'answer': 'Los Angeles'}\n",
            "Question 4. {'score': 0.3695431649684906, 'start': 387, 'end': 396, 'answer': 'Las Vegas'}\n",
            "Question 5. {'score': 0.21839778125286102, 'start': 355, 'end': 362, 'answer': 'Barstow'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TPd42T7TrhVH"
      },
      "source": [
        "Question-answering applied to NER person entities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6yQyrSjsv6dJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4780dda5-4485-417e-c0e1-1c4ca8bd9cb5"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')\n",
        "nlp_qa(context=sequence, question='Who was singing ?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Nat King Cole',\n",
              " 'end': 277,\n",
              " 'score': 0.9653680324554443,\n",
              " 'start': 264}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CfOlUtS0wapC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4cce8650-5d46-4374-d987-8111c6e81cbc"
      },
      "source": [
        "nlp_qa(context=sequence, question='Who was going to Las Vegas ?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Nat King Cole',\n",
              " 'end': 277,\n",
              " 'score': 0.4316245913505554,\n",
              " 'start': 264}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DI_8OcAdx7Rp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "212ae5d4-23e1-4016-e846-fe74cef78a26"
      },
      "source": [
        "nlp_qa(context=sequence, question='Who are they?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Jo and Maria',\n",
              " 'end': 305,\n",
              " 'score': 0.8486908078193665,\n",
              " 'start': 293}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oc3Pe7CByyhc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5625a1f6-1b12-4ab4-813b-74a0fa3f727d"
      },
      "source": [
        "nlp_qa(context=sequence, question='Who drove to Las Vegas?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Nat King Cole was singing as Jo and Maria',\n",
              " 'end': 305,\n",
              " 'score': 0.35941559076309204,\n",
              " 'start': 264}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TmF96wthzwWT"
      },
      "source": [
        "Description of the Default Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_EMgV9dnz60s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "86ed2cfb-d7e1-4038-fd6a-daba48a80414"
      },
      "source": [
        "print(nlp_qa.model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "DistilBertForQuestionAnswering(\n",
            "  (distilbert): DistilBertModel(\n",
            "    (embeddings): Embeddings(\n",
            "      (word_embeddings): Embedding(28996, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (transformer): Transformer(\n",
            "      (layer): ModuleList(\n",
            "        (0): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (1): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (2): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (3): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (4): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (5): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n",
            "  (dropout): Dropout(p=0.1, inplace=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vAlWY5E6TfKL"
      },
      "source": [
        "Question-Answering with ELECTRA"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BFNSvGN0znq9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 386,
          "referenced_widgets": [
            "ec5480ed053b46cdb517d77899900a2f",
            "85a1571687b84d19ae442c5f81f26f7a",
            "e2eb771a977f44c180f99f87ca99fd77",
            "cb3ee57a490d4a9592e4b122d0d81948",
            "3da87e6c040b440988d93c43ac3a2c09",
            "dceac99c8e52482e9f62a5a55898641e",
            "a07d84c9acd24a909d65f8b16f85fbe9",
            "1f9e54937bcc49858ae9a938d899379b",
            "27a07215928f497db5e317b82e9e5922",
            "7c2581de98954b79b19ee3c6a2259ba7",
            "7b616c40df534e3fa921221fe620a3d9",
            "23afbc66e2dd43eb84d3d731c46263f1",
            "e084d2d75eb14fe3aacedbd5ecf711bc",
            "a4b121058c934cf081e8af64e893b913",
            "b7c0ba54037049e7a9ce91a85c41c580",
            "d7d6eb6e2945450fb1c0590a506222e5",
            "a7f35783ec6249be8ccfba1c83ed0e9f",
            "c9bd7a41ada546e88509a60576dcbd81",
            "dad65f326c614a58a0c13c94accab562",
            "f853299184d54861874b30a6087c6e3b",
            "863fcf057e7d4fe984c531d6b1291814",
            "058caba18b6d4a1e8fb52d573465255e",
            "6234b7e5ca9c4067ae359a31f9b38e27",
            "a475b921ef3d4315990e07512ff759a3",
            "469aaef964d644198b9cf9b878c56178",
            "045cce4661714b078350aa8c12f86680",
            "4f43fcc08b664d0d9a9c5edbec57dfe2",
            "a61f3ca504574a4db912806d920daad9",
            "da55a1ee4c1547c180fc2fa62e8908d2",
            "9d754136f43641b4996383d99a3163c3",
            "fd4c6744a60340b4b004710cf2e9c96c",
            "3a196782e8624fca9baf0561b48cf0b8"
          ]
        },
        "outputId": "1f9b1b3e-f51d-48dd-a97e-d5f43d3207c9"
      },
      "source": [
        "nlp_qa = pipeline('question-answering', model='google/electra-small-generator', tokenizer='google/electra-small-generator')\n",
        "nlp_qa(context=sequence, question='Who drove to Las Vegas ?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ec5480ed053b46cdb517d77899900a2f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=463.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "27a07215928f497db5e317b82e9e5922",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=54236116.0, style=ProgressStyle(descrip…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a7f35783ec6249be8ccfba1c83ed0e9f",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "469aaef964d644198b9cf9b878c56178",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at google/electra-small-generator were not used when initializing ElectraForQuestionAnswering: ['generator_predictions.LayerNorm.weight', 'generator_predictions.LayerNorm.bias', 'generator_predictions.dense.weight', 'generator_predictions.dense.bias', 'generator_lm_head.weight', 'generator_lm_head.bias']\n",
            "- This IS expected if you are initializing ElectraForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing ElectraForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of ElectraForQuestionAnswering were not initialized from the model checkpoint at google/electra-small-generator and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'rather pleasant to be making it out of the city on this',\n",
              " 'end': 245,\n",
              " 'score': 0.00034621506347320974,\n",
              " 'start': 190}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "49PDRpKHsc41"
      },
      "source": [
        "Question Answering with default Model and SRL"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W8kGz5ihz96g",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "393a5a3e-ea75-4c7e-84f3-9f9930edd164"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')\n",
        "nlp_qa(context=sequence, question='What was slow?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'The traffic', 'end': 11, 'score': 0.46530455350875854, 'start': 0}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4mycOJhdugbL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "aeeb1e97-51e7-4658-da75-cbeba382521b"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')\n",
        "nlp_qa(context=sequence, question='What was playing')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'cool jazz', 'end': 152, 'score': 0.3511938154697418, 'start': 143}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bniJUNoxwtiw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b3ea4b39-8c69-4cef-9a57-401aefd9065e"
      },
      "source": [
        "nlp_qa = pipeline('question-answering')\n",
        "nlp_qa(context=sequence, question='Who sees a show?')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer': 'Nat King Cole',\n",
              " 'end': 277,\n",
              " 'score': 0.5588219165802002,\n",
              " 'start': 264}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    }
  ]
}